#!/local/usr/bin/python

import os, sys
import cnl2mc_console

class Validator:
    
    def __init__(self):
        self.console = cnl2mc_console.Console()
    def config(self, configFile):
        self.console.config(configFile)
    
    def splitIdIntoTestingFoldsDict(self, dataNodeDict, n = 10):
        total = len(dataNodeDict)
        foldSize = total / n
        testingIdFoldsDict = {}
        ids = dataNodeDict.keys()
        import random
        random.seed(1)
        for i in range(n):
            testingIdFold = random.sample(ids, foldSize)
            testingIdFoldsDict[i] = testingIdFold
            for id in testingIdFold:
                ids.remove(id)
        return testingIdFoldsDict
            
    def getTestingAndTrainingFold(self, dataNodeDict, testingIdFold):
        testingNodesDict = {}
        trainingNodesDict = dataNodeDict.copy()
        for key in testingIdFold:
            testingNodesDict[key] = trainingNodesDict.pop(key)
        return (testingNodesDict, trainingNodesDict)
    
    def main(self):
        console = self.console
        sqlConfigDict = console.xmlConf.getSqlConfig()
        dataNodeDict = console.readTrainingDataNodeDict(sqlConfigDict)                                            
        console.preprocess(dataNodeDict)
        testingFoldsDict = self.splitIdIntoTestingFoldsDict(dataNodeDict, 10)        
        for (k, testingFold) in testingFoldsDict.items():
            (testingNodeDict, trainingNodeDict) = self.getTestingAndTrainingFold(dataNodeDict, testingFold)
            
            dirname = console.xmlConf.getGenerateFilePath()           
            basename = os.path.basename(console.xmlConf.getFeatureMatrixFileName())
            outputPath = os.path.join(dirname,str(k))
            trainingFeatureMatrixFileName = os.path.join(outputPath, "train_"+basename)
            testingFeatureMatrixFileName = os.path.join(outputPath, "test_"+basename)
            
            indexMethod = console.xmlConf.getIndexMethod()
            indexer = console.instFact.getNewIndexer(indexMethod)
            attachment = console.getIndexAttachment(indexer, trainingNodeDict)
            console.indexInFile(trainingFeatureMatrixFileName, trainingNodeDict, indexer, attachment)
            console.indexInFile(testingFeatureMatrixFileName, testingNodeDict, indexer, attachment)                                
            focusedClassesFilename = console.xmlConf.getFocusedClassesFilename()            
            """
            # <exp>
            import histogram
            reportName = "../experiments/globalClassFreqDist.txt"
            histogram.reportFreqDistHistogram(console.globalClassFreqDist, reportName)
            print "wordAndClassHistogram...\n"
            focusedClassList = open(focusedClassesFilename, 'rU').read().split()
            trainingDataNodesIter = trainingNodeDict.itervalues()
            histogram.wordAndClassHistogram(console.globalWordFreqDist, focusedClassList, trainingDataNodesIter)
            # </exp>
            """      
            
            """ """
            if os.path.exists(focusedClassesFilename):
                focusedClassList = open(focusedClassesFilename, 'rU').read().split()
            else:
                focusedClassList = console.globalClassFreqDist.keys()        
            totalFileNumber = len(focusedClassList)
            printedFileNumber = 0
            print "totalFileNumber: %d"%totalFileNumber
            
            toFileType = console.xmlConf.getGenerateFileType()
                        
            reduceDimMethodType = console.xmlConf.getReduceDimMethodType()
            reduceDimMethodName = console.xmlConf.getReduceDimMethodName()
            freqThreshold = console.xmlConf.getFrequencyThreshold()
            IgThreshold = console.xmlConf.getInfoGainThreshold()
            
            for pickedClass in focusedClassList:
                binaryClassList = [pickedClass, '-1']
                reducedDimWordList = console.reduceDim(trainingNodeDict, binaryClassList, reduceDimMethodType, reduceDimMethodName, freqThreshold, IgThreshold)
                trainingFilename = "train_%s"%pickedClass
                # add IG_Reducing here. And a arff attributes parser
                testingFilename =  "test_%s"%pickedClass
                console.generateVectorFile(trainingNodeDict, trainingFeatureMatrixFileName, reducedDimWordList, toFileType, binaryClassList, outputPath, trainingFilename)
                console.generateVectorFile(testingNodeDict, testingFeatureMatrixFileName, reducedDimWordList, toFileType, binaryClassList, outputPath, testingFilename)
                printedFileNumber += 1
                print " %d file(s) in process: now is %d, -- %d file(s) left.\n" \
                        %(totalFileNumber, printedFileNumber, totalFileNumber - printedFileNumber)
                        
            os.remove(trainingFeatureMatrixFileName)
            os.remove(testingFeatureMatrixFileName)
            """ """
if __name__ == '__main__':    
    validator = Validator()
    configFile = sys.argv[1]
    validator.config(configFile)
    
    validator.main()
    
    
    
    
    
    
    
    
    
    
    